{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "\n",
    "# Set OpenAI's API key and API base to use vLLM's API server.\n",
    "openai_api_key = \"EMPTY\"\n",
    "openai_api_base = \"http://localhost:8001/v1\"\n",
    "\n",
    "client = OpenAI(\n",
    "    api_key=openai_api_key,\n",
    "    base_url=openai_api_base,\n",
    ")\n",
    "\n",
    "chat_response = client.chat.completions.create(\n",
    "    model=\"Qwen1.5-7B-Chat\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "        {\"role\": \"user\", \"content\": \"1+1=几\"},\n",
    "    ],\n",
    ")\n",
    "print(\"Chat response:\\n\\n\", chat_response.choices[0].message.content[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leading_text = \"{\"\n",
    "messages = [\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"content\": \"你是人工智能助手，你更擅长中文和英文的对话。你会为用户提供安全，有帮助，准确的回答。\",\n",
    "    },\n",
    "    {\n",
    "        \"role\": \"user\",\n",
    "        \"content\": '你是一个情景分析专家，用来分析一句短语中的主语故事中的人物、人物心理和下一步行为预测。你需要阅读完所有的故事，然后输出一个 JSON 格式便于后续的处理。请严格参考 #sample output 格式输出一个合法 JSON 字符\\n\\n#example:\\nsentence: You can look up the word in the dictionary.\\n动词短语: look up\\n\\n#pos_list\\n\"副词\", \"连词\", \"动词短语\"\\n#sample output:\\n[\\n {\\n \"index\": \"copy [sentence index] value from sentence\", \\n \"sentence\": \"*\", \\n \"keywords\": \"*\", \\n \"POS_tagging\": \"*\", \\n }\\n]\\n\\n#input:\\n桃园三结义#output:\\n ',\n",
    "    },\n",
    "    {\"role\": \"assistant\", \"content\": leading_text},\n",
    "]\n",
    "chat_response = client.chat.completions.create(\n",
    "    model=\"Qwen1.5-7B-Chat\",\n",
    "    messages=messages,\n",
    "    add_generation_prompt=False\n",
    ")\n",
    "print(\"Chat response:\\n\\n\", chat_response.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "ConnectionError",
     "evalue": "('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRemoteDisconnected\u001b[0m                        Traceback (most recent call last)",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/connectionpool.py:714\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    713\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 714\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_request(\n\u001b[1;32m    715\u001b[0m     conn,\n\u001b[1;32m    716\u001b[0m     method,\n\u001b[1;32m    717\u001b[0m     url,\n\u001b[1;32m    718\u001b[0m     timeout\u001b[38;5;241m=\u001b[39mtimeout_obj,\n\u001b[1;32m    719\u001b[0m     body\u001b[38;5;241m=\u001b[39mbody,\n\u001b[1;32m    720\u001b[0m     headers\u001b[38;5;241m=\u001b[39mheaders,\n\u001b[1;32m    721\u001b[0m     chunked\u001b[38;5;241m=\u001b[39mchunked,\n\u001b[1;32m    722\u001b[0m )\n\u001b[1;32m    724\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    725\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    726\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    727\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/connectionpool.py:466\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    462\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    463\u001b[0m             \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    464\u001b[0m             \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    465\u001b[0m             \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m--> 466\u001b[0m             six\u001b[38;5;241m.\u001b[39mraise_from(e, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    467\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/connectionpool.py:461\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    460\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 461\u001b[0m     httplib_response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[1;32m    462\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    463\u001b[0m     \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    464\u001b[0m     \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    465\u001b[0m     \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/http/client.py:1378\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1377\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1378\u001b[0m     response\u001b[38;5;241m.\u001b[39mbegin()\n\u001b[1;32m   1379\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/http/client.py:318\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 318\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_status()\n\u001b[1;32m    319\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/http/client.py:287\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    284\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m line:\n\u001b[1;32m    285\u001b[0m     \u001b[38;5;66;03m# Presumably, the server closed the connection before\u001b[39;00m\n\u001b[1;32m    286\u001b[0m     \u001b[38;5;66;03m# sending a valid response.\u001b[39;00m\n\u001b[0;32m--> 287\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m RemoteDisconnected(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRemote end closed connection without\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    288\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m response\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    289\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[0;31mRemoteDisconnected\u001b[0m: Remote end closed connection without response",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mProtocolError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/requests/adapters.py:486\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    485\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 486\u001b[0m     resp \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39murlopen(\n\u001b[1;32m    487\u001b[0m         method\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mmethod,\n\u001b[1;32m    488\u001b[0m         url\u001b[38;5;241m=\u001b[39murl,\n\u001b[1;32m    489\u001b[0m         body\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mbody,\n\u001b[1;32m    490\u001b[0m         headers\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mheaders,\n\u001b[1;32m    491\u001b[0m         redirect\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    492\u001b[0m         assert_same_host\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    493\u001b[0m         preload_content\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    494\u001b[0m         decode_content\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    495\u001b[0m         retries\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_retries,\n\u001b[1;32m    496\u001b[0m         timeout\u001b[38;5;241m=\u001b[39mtimeout,\n\u001b[1;32m    497\u001b[0m         chunked\u001b[38;5;241m=\u001b[39mchunked,\n\u001b[1;32m    498\u001b[0m     )\n\u001b[1;32m    500\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/connectionpool.py:798\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    796\u001b[0m     e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[0;32m--> 798\u001b[0m retries \u001b[38;5;241m=\u001b[39m retries\u001b[38;5;241m.\u001b[39mincrement(\n\u001b[1;32m    799\u001b[0m     method, url, error\u001b[38;5;241m=\u001b[39me, _pool\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m, _stacktrace\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mexc_info()[\u001b[38;5;241m2\u001b[39m]\n\u001b[1;32m    800\u001b[0m )\n\u001b[1;32m    801\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/util/retry.py:550\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    549\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_method_retryable(method):\n\u001b[0;32m--> 550\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m six\u001b[38;5;241m.\u001b[39mreraise(\u001b[38;5;28mtype\u001b[39m(error), error, _stacktrace)\n\u001b[1;32m    551\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/packages/six.py:769\u001b[0m, in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m    768\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m value\u001b[38;5;241m.\u001b[39m__traceback__ \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m tb:\n\u001b[0;32m--> 769\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m value\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m    770\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m value\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/connectionpool.py:714\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    713\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 714\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_request(\n\u001b[1;32m    715\u001b[0m     conn,\n\u001b[1;32m    716\u001b[0m     method,\n\u001b[1;32m    717\u001b[0m     url,\n\u001b[1;32m    718\u001b[0m     timeout\u001b[38;5;241m=\u001b[39mtimeout_obj,\n\u001b[1;32m    719\u001b[0m     body\u001b[38;5;241m=\u001b[39mbody,\n\u001b[1;32m    720\u001b[0m     headers\u001b[38;5;241m=\u001b[39mheaders,\n\u001b[1;32m    721\u001b[0m     chunked\u001b[38;5;241m=\u001b[39mchunked,\n\u001b[1;32m    722\u001b[0m )\n\u001b[1;32m    724\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    725\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    726\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    727\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/connectionpool.py:466\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    462\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    463\u001b[0m             \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    464\u001b[0m             \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    465\u001b[0m             \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m--> 466\u001b[0m             six\u001b[38;5;241m.\u001b[39mraise_from(e, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    467\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/urllib3/connectionpool.py:461\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    460\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 461\u001b[0m     httplib_response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[1;32m    462\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    463\u001b[0m     \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    464\u001b[0m     \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    465\u001b[0m     \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/http/client.py:1378\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1377\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1378\u001b[0m     response\u001b[38;5;241m.\u001b[39mbegin()\n\u001b[1;32m   1379\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/http/client.py:318\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 318\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_status()\n\u001b[1;32m    319\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/http/client.py:287\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    284\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m line:\n\u001b[1;32m    285\u001b[0m     \u001b[38;5;66;03m# Presumably, the server closed the connection before\u001b[39;00m\n\u001b[1;32m    286\u001b[0m     \u001b[38;5;66;03m# sending a valid response.\u001b[39;00m\n\u001b[0;32m--> 287\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m RemoteDisconnected(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRemote end closed connection without\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    288\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m response\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    289\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[0;31mProtocolError\u001b[0m: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[20], line 51\u001b[0m\n\u001b[1;32m      4\u001b[0m messages \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m      5\u001b[0m     {\n\u001b[1;32m      6\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msystem\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     13\u001b[0m     {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massistant\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m: leading_text},\n\u001b[1;32m     14\u001b[0m ]\n\u001b[1;32m     15\u001b[0m data \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mQwen1.5-7B-Chat\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     17\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m: messages,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mguided_choice\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m     48\u001b[0m }\n\u001b[0;32m---> 51\u001b[0m web \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mpost(url\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://localhost:8001/v1/chat/completions\u001b[39m\u001b[38;5;124m\"\u001b[39m, json\u001b[38;5;241m=\u001b[39mdata)\n\u001b[1;32m     52\u001b[0m web\u001b[38;5;241m.\u001b[39mstatus_code\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/requests/api.py:115\u001b[0m, in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(url, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, json\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    104\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[1;32m    105\u001b[0m \n\u001b[1;32m    106\u001b[0m \u001b[38;5;124;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[38;5;124;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m    113\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 115\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m request(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpost\u001b[39m\u001b[38;5;124m\"\u001b[39m, url, data\u001b[38;5;241m=\u001b[39mdata, json\u001b[38;5;241m=\u001b[39mjson, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m session\u001b[38;5;241m.\u001b[39mrequest(method\u001b[38;5;241m=\u001b[39mmethod, url\u001b[38;5;241m=\u001b[39murl, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    585\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m    586\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m    587\u001b[0m }\n\u001b[1;32m    588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msend(prep, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39msend_kwargs)\n\u001b[1;32m    591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m adapter\u001b[38;5;241m.\u001b[39msend(request, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m    706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n",
      "File \u001b[0;32m~/anaconda3/envs/vllm/lib/python3.11/site-packages/requests/adapters.py:501\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    486\u001b[0m     resp \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39murlopen(\n\u001b[1;32m    487\u001b[0m         method\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mmethod,\n\u001b[1;32m    488\u001b[0m         url\u001b[38;5;241m=\u001b[39murl,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    497\u001b[0m         chunked\u001b[38;5;241m=\u001b[39mchunked,\n\u001b[1;32m    498\u001b[0m     )\n\u001b[1;32m    500\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 501\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(err, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m    503\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m MaxRetryError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    504\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, ConnectTimeoutError):\n\u001b[1;32m    505\u001b[0m         \u001b[38;5;66;03m# TODO: Remove this in 3.0.0: see #2811\u001b[39;00m\n",
      "\u001b[0;31mConnectionError\u001b[0m: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "leading_text = \"{\"\n",
    "messages = [\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"content\": \"你是人工智能助手，你更擅长中文和英文的对话。你会为用户提供安全，有帮助，准确的回答。\",\n",
    "    },\n",
    "    {\n",
    "        \"role\": \"user\",\n",
    "        \"content\": '你是一个情景分析专家，用来分析一句短语中的主语故事中的人物、人物心理和下一步行为预测。你需要阅读完所有的故事，然后输出一个 JSON 格式便于后续的处理。请严格参考 #sample output 格式输出一个合法 JSON 字符\\n\\n#example:\\nsentence: You can look up the word in the dictionary.\\n动词短语: look up\\n\\n#pos_list\\n\"副词\", \"连词\", \"动词短语\"\\n#sample output:\\n[\\n {\\n \"index\": \"copy [sentence index] value from sentence\", \\n \"sentence\": \"*\", \\n \"keywords\": \"*\", \\n \"POS_tagging\": \"*\", \\n }\\n]\\n\\n#input:\\n桃园三结义#output:\\n ',\n",
    "    },\n",
    "    {\"role\": \"assistant\", \"content\": leading_text},\n",
    "]\n",
    "data = {\n",
    "    \"model\": \"Qwen1.5-7B-Chat\",\n",
    "    \"messages\": messages,\n",
    "    \"temperature\": 0.7,\n",
    "    \"top_p\": 1.0,\n",
    "    \"n\": 1,\n",
    "    \"max_tokens\": None,\n",
    "    \"seed\": None,\n",
    "    \"stop\": [],\n",
    "    \"stream\": False,\n",
    "    \"logprobs\": False,\n",
    "    \"top_logprobs\": None,\n",
    "    \"presence_penalty\": 0.0,\n",
    "    \"frequency_penalty\": 0.0,\n",
    "    \"logit_bias\": None,\n",
    "    \"user\": None,\n",
    "    \"best_of\": None,\n",
    "    \"top_k\": -1,\n",
    "    \"ignore_eos\": False,\n",
    "    \"use_beam_search\": False,\n",
    "    \"early_stopping\": False,\n",
    "    \"stop_token_ids\": [],\n",
    "    \"skip_special_tokens\": True,\n",
    "    \"spaces_between_special_tokens\": True,\n",
    "    \"add_generation_prompt\": False,\n",
    "    \"echo\": False,\n",
    "    \"repetition_penalty\": 1.0,\n",
    "    \"min_p\": 0.0,\n",
    "    \"include_stop_str_in_output\": False,\n",
    "    \"length_penalty\": 1.0,\n",
    "    \"guided_json\": None,\n",
    "    \"guided_regex\": None,\n",
    "    \"guided_choice\": None,\n",
    "}\n",
    "\n",
    "\n",
    "web = requests.post(url=\"http://localhost:8001/v1/chat/completions\", json=data)\n",
    "web.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'cmpl-67d3f2f4aec3450e953de34e8c4e5dbf',\n",
       " 'object': 'chat.completion',\n",
       " 'created': 4520,\n",
       " 'model': 'Qwen1.5-7B-Chat',\n",
       " 'choices': [{'index': 0,\n",
       "   'message': {'role': 'assistant',\n",
       "    'content': ' \"index\": \"copy \\'桃园三结义\\' value from sentence\", \\n \"sentence\": \"桃园三结义\", \\n \"keywords\": [\"桃园\", \"三\", \"结义\", \"人物\", \"人物心理\", \"下一步行为预测\", \"主语故事\"],\\n \"POS_tagging\": [\\n {\\n \"index\": \"名词\", \"word\": \"桃园\",\\n \"POS\": \"NN\"\\n },\\n {\\n \"index\": \"动词短语\", \"word\": \"结义\",\\n \"POS\": \"VBD\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"三\",\\n \"POS\": \"NN\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"结义\",\\n \"POS\": \"VBD\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"人物\", \"POS\": \"NNS\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"心理\", \"POS\": \"NNS\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"下一步行为预测\", \"POS\": \"VBD\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"主语故事\", \"POS\": \"NNS\"\\n }\\n ]\\n}'},\n",
       "   'logprobs': None,\n",
       "   'finish_reason': 'stop'}],\n",
       " 'usage': {'prompt_tokens': 189,\n",
       "  'total_tokens': 433,\n",
       "  'completion_tokens': 244}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "web.json(\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' \"index\": \"copy \\'桃园三结义\\' value from sentence\", \\n \"sentence\": \"桃园三结义\", \\n \"keywords\": [\"桃园\", \"三\", \"结义\", \"人物\", \"人物心理\", \"下一步行为预测\", \"主语故事\"],\\n \"POS_tagging\": [\\n {\\n \"index\": \"名词\", \"word\": \"桃园\",\\n \"POS\": \"NN\"\\n },\\n {\\n \"index\": \"动词短语\", \"word\": \"结义\",\\n \"POS\": \"VBD\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"三\",\\n \"POS\": \"NN\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"结义\",\\n \"POS\": \"VBD\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"人物\", \"POS\": \"NNS\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"心理\", \"POS\": \"NNS\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"下一步行为预测\", \"POS\": \"VBD\"\\n },\\n {\\n \"index\": \"名词\", \"word\": \"主语故事\", \"POS\": \"NNS\"\\n }\\n ]\\n}'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "web.json().get('choices')[0].get('message').get('content')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'index': \"copy '桃园三结义' value from sentence\",\n",
       " 'sentence': '桃园三结义',\n",
       " 'keywords': ['桃园', '三', '结义', '人物', '人物心理', '下一步行为预测', '主语故事'],\n",
       " 'POS_tagging': [{'index': '名词', 'word': '桃园', 'POS': 'NN'},\n",
       "  {'index': '动词短语', 'word': '结义', 'POS': 'VBD'},\n",
       "  {'index': '名词', 'word': '三', 'POS': 'NN'},\n",
       "  {'index': '名词', 'word': '结义', 'POS': 'VBD'},\n",
       "  {'index': '名词', 'word': '人物', 'POS': 'NNS'},\n",
       "  {'index': '名词', 'word': '心理', 'POS': 'NNS'},\n",
       "  {'index': '名词', 'word': '下一步行为预测', 'POS': 'VBD'},\n",
       "  {'index': '名词', 'word': '主语故事', 'POS': 'NNS'}]}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "json.loads(\"{\" + web.json().get(\"choices\")[0].get(\"message\").get(\"content\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vllm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
